package scheduler;

import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.scheduler.PriorityScheduler;
import us.codecraft.webmagic.scheduler.RedisPriorityScheduler;
import us.codecraft.webmagic.utils.HttpConstant;

public class ReCrawlRedisPriorityScheduler extends RedisPriorityScheduler {
    public ReCrawlRedisPriorityScheduler(String host) {
        super(host);
    }

    @Override
    protected boolean noNeedToRemoveDuplicate(Request request) {
        boolean a = request.getExtras().containsKey("noNeedToRemoveDuplicate") ;
        boolean b = HttpConstant.Method.POST.equalsIgnoreCase(request.getMethod());
        return request.getExtras().containsKey("noNeedToRemoveDuplicate") || HttpConstant.Method.POST.equalsIgnoreCase(request.getMethod());
    }
}
